In [1]:
from google.colab import drive
drive.mount('/content/drive')
Mounted at /content/drive
In [2]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
import tensorflow as tf
import cv2
import keras
from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
from keras.preprocessing.image import array_to_img
In [3]:
train_path ='/content/drive/MyDrive/Capstone Data/Train Images'
test_path ='/content/drive/MyDrive/Capstone Data/Test Images'
In [4]:
def display_images():
  category = os.listdir(train_path)
  for category in category:
        image = load_img(os.path.join(train_path,category,os.listdir(os.path.join(train_path,category))[0]))
        image = image.resize([100,100])
        print('category:',category)
        plt.imshow(image)
        plt.show()

display_images()
category: Acura TL Type-S 2008
category: AM General Hummer SUV 2000
category: Acura ZDX Hatchback 2012
category: Aston Martin V8 Vantage Convertible 2012
category: Acura RL Sedan 2012
category: Aston Martin Virage Convertible 2012
category: Acura TL Sedan 2012
category: Acura TSX Sedan 2012
category: Aston Martin V8 Vantage Coupe 2012
category: Acura Integra Type R 2001
category: Aston Martin Virage Coupe 2012
category: Audi S5 Convertible 2012
category: Audi 100 Wagon 1994
category: Audi S4 Sedan 2007
category: Audi A5 Coupe 2012
category: Audi 100 Sedan 1994
category: Audi S5 Coupe 2012
category: Audi R8 Coupe 2012
category: Audi RS 4 Convertible 2008
category: Audi S4 Sedan 2012
category: Audi S6 Sedan 2011
category: Bentley Mulsanne Sedan 2011
category: Bentley Arnage Sedan 2009
category: Audi TT Hatchback 2011
category: Audi TTS Coupe 2012
category: Bentley Continental GT Coupe 2007
category: Audi V8 Sedan 1994
category: Bentley Continental Flying Spur Sedan 2007
category: Bentley Continental Supersports Conv. Convertible 2012
category: Bentley Continental GT Coupe 2012
category: Audi TT RS Coupe 2012
category: BMW 1 Series Convertible 2012
category: BMW X5 SUV 2007
category: BMW X6 SUV 2012
category: BMW 6 Series Convertible 2007
category: BMW M6 Convertible 2010
category: BMW X3 SUV 2012
category: BMW 3 Series Sedan 2012
category: BMW M5 Sedan 2010
category: BMW ActiveHybrid 5 Sedan 2012
category: BMW 1 Series Coupe 2012
category: BMW 3 Series Wagon 2012
category: BMW M3 Coupe 2012
category: BMW Z4 Convertible 2012
category: Cadillac Escalade EXT Crew Cab 2007
category: Bugatti Veyron 16.4 Convertible 2009
category: Cadillac SRX SUV 2012
category: Buick Verano Sedan 2012
category: Buick Enclave SUV 2012
category: Cadillac CTS-V Sedan 2012
category: Bugatti Veyron 16.4 Coupe 2009
category: Buick Regal GS 2012
category: Buick Rainier SUV 2007
category: Chevrolet Avalanche Crew Cab 2012
category: Chevrolet Corvette Convertible 2012
category: Chevrolet Corvette ZR1 2012
category: Chevrolet Express Van 2007
category: Chevrolet Corvette Ron Fellows Edition Z06 2007
category: Chevrolet Express Cargo Van 2007
category: Chevrolet Impala Sedan 2007
category: Chevrolet Cobalt SS 2010
category: Chevrolet Camaro Convertible 2012
category: Chevrolet HHR SS 2010
category: Chevrolet Malibu Hybrid Sedan 2010
category: Chevrolet TrailBlazer SS 2009
category: Chevrolet Silverado 1500 Regular Cab 2012
category: Chevrolet Malibu Sedan 2007
category: Chevrolet Tahoe Hybrid SUV 2012
category: Chevrolet Silverado 1500 Extended Cab 2012
category: Chevrolet Sonic Sedan 2012
category: Chevrolet Silverado 1500 Classic Extended Cab 2007
category: Chevrolet Silverado 2500HD Regular Cab 2012
category: Chevrolet Silverado 1500 Hybrid Crew Cab 2012
category: Chevrolet Monte Carlo Coupe 2007
category: Chevrolet Traverse SUV 2012
category: Chrysler PT Cruiser Convertible 2008
category: Chrysler Town and Country Minivan 2012
category: Chrysler Aspen SUV 2009
category: Dodge Caliber Wagon 2012
category: Daewoo Nubira Wagon 2002
category: Dodge Caliber Wagon 2007
category: Chrysler Crossfire Convertible 2008
category: Chrysler Sebring Convertible 2010
category: Chrysler 300 SRT-8 2010
category: Dodge Caravan Minivan 1997
category: Dodge Charger Sedan 2012
category: Dodge Journey SUV 2012
category: Dodge Durango SUV 2012
category: Dodge Durango SUV 2007
category: Dodge Magnum Wagon 2008
category: Dodge Dakota Crew Cab 2010
category: Dodge Charger SRT-8 2009
category: Dodge Challenger SRT8 2011
category: Dodge Dakota Club Cab 2007
category: Dodge Ram Pickup 3500 Crew Cab 2010
category: Eagle Talon Hatchback 1998
category: FIAT 500 Convertible 2012
category: Dodge Sprinter Cargo Van 2009
category: Ferrari FF Coupe 2012
category: FIAT 500 Abarth 2012
category: Dodge Ram Pickup 3500 Quad Cab 2009
category: Ferrari 458 Italia Coupe 2012
category: Ferrari California Convertible 2012
category: Ferrari 458 Italia Convertible 2012
category: Fisker Karma Sedan 2012
category: Ford Fiesta Sedan 2012
category: Ford Expedition EL SUV 2009
category: Ford F-150 Regular Cab 2007
category: Ford Freestar Minivan 2007
category: Ford F-150 Regular Cab 2012
category: Ford Focus Sedan 2007
category: Ford F-450 Super Duty Crew Cab 2012
category: Ford Edge SUV 2012
category: Ford E-Series Wagon Van 2012
category: Ford GT Coupe 2006
category: GMC Savana Van 2012
category: GMC Yukon Hybrid SUV 2012
category: GMC Acadia SUV 2012
category: Geo Metro Convertible 1993
category: GMC Canyon Extended Cab 2012
category: Ford Ranger SuperCab 2011
category: Ford Mustang Convertible 2007
category: Honda Accord Coupe 2012
category: GMC Terrain SUV 2012
category: Honda Accord Sedan 2012
category: Hyundai Accent Sedan 2012
category: HUMMER H3T Crew Cab 2010
category: Hyundai Genesis Sedan 2012
category: Hyundai Elantra Sedan 2007
category: HUMMER H2 SUT Crew Cab 2009
category: Honda Odyssey Minivan 2007
category: Honda Odyssey Minivan 2012
category: Hyundai Elantra Touring Hatchback 2012
category: Hyundai Azera Sedan 2012
category: Hyundai Santa Fe SUV 2012
category: Infiniti G Coupe IPL 2012
category: Infiniti QX56 SUV 2011
category: Isuzu Ascender SUV 2008
category: Jaguar XK XKR 2012
category: Hyundai Veracruz SUV 2012
category: Hyundai Sonata Sedan 2012
category: Hyundai Sonata Hybrid Sedan 2012
category: Hyundai Tucson SUV 2012
category: Hyundai Veloster Hatchback 2012
category: Jeep Compass SUV 2012
category: Lamborghini Gallardo LP 570-4 Superleggera 2012
category: Jeep Grand Cherokee SUV 2012
category: Lamborghini Reventon Coupe 2008
category: Land Rover LR2 SUV 2012
category: Jeep Patriot SUV 2012
category: Jeep Wrangler SUV 2012
category: Jeep Liberty SUV 2012
category: Lamborghini Aventador Coupe 2012
category: Lamborghini Diablo Coupe 2001
category: Land Rover Range Rover SUV 2012
category: Lincoln Town Car Sedan 2011
category: Maybach Landaulet Convertible 2012
category: Mercedes-Benz C-Class Sedan 2012
category: McLaren MP4-12C Coupe 2012
category: Mercedes-Benz S-Class Sedan 2012
category: Mercedes-Benz E-Class Sedan 2012
category: Mercedes-Benz SL-Class Coupe 2009
category: Mazda Tribute SUV 2011
category: Mercedes-Benz 300-Class Convertible 1993
category: Mercedes-Benz Sprinter Van 2012
category: MINI Cooper Roadster Convertible 2012
category: Nissan Leaf Hatchback 2012
category: Nissan Juke Hatchback 2012
category: Rolls-Royce Ghost Sedan 2012
category: Nissan NV Passenger Van 2012
category: Nissan 240SX Coupe 1998
category: Plymouth Neon Coupe 1999
category: Mitsubishi Lancer Sedan 2012
category: Porsche Panamera Sedan 2012
category: Ram C-V Cargo Van Minivan 2012
category: Rolls-Royce Phantom Drophead Coupe Convertible 2012
category: Rolls-Royce Phantom Sedan 2012
category: Tesla Model S Sedan 2012
category: smart fortwo Convertible 2012
category: Suzuki SX4 Hatchback 2012
category: Spyker C8 Convertible 2009
category: Scion xD Hatchback 2012
category: Suzuki SX4 Sedan 2012
category: Suzuki Aerio Sedan 2007
category: Spyker C8 Coupe 2009
category: Suzuki Kizashi Sedan 2012
category: Toyota 4Runner SUV 2012
category: Toyota Camry Sedan 2012
category: Volkswagen Beetle Hatchback 2012
category: Volvo C30 Hatchback 2012
category: Toyota Corolla Sedan 2012
category: Toyota Sequoia SUV 2012
category: Volvo 240 Sedan 1993
category: Volkswagen Golf Hatchback 2012
category: Volkswagen Golf Hatchback 1991
category: Volvo XC90 SUV 2007
In [5]:
train_df = pd.read_csv('/content/drive/MyDrive/Capstone Data/Annotations/Train Annotations.csv',header = None,names = ['file','x_min','y_min','x_max','y_max','class'])
In [6]:
train_df
Out[6]:
file x_min y_min x_max y_max class
0 00001.jpg 39 116 569 375 14
1 00002.jpg 36 116 868 587 3
2 00003.jpg 85 109 601 381 91
3 00004.jpg 621 393 1484 1096 134
4 00005.jpg 14 36 133 99 106
... ... ... ... ... ... ...
8139 08140.jpg 3 44 423 336 78
8140 08141.jpg 138 150 706 523 196
8141 08142.jpg 26 246 660 449 163
8142 08143.jpg 78 526 1489 908 112
8143 08144.jpg 20 240 862 677 17

8144 rows × 6 columns

In [7]:
test_df = pd.read_csv('/content/drive/MyDrive/Capstone Data/Annotations/Test Annotation.csv',header = None,names = ['file','x_min','y_min','x_max','y_max','class'])
In [8]:
test_df
Out[8]:
file x_min y_min x_max y_max class
0 00001.jpg 30 52 246 147 181
1 00002.jpg 100 19 576 203 103
2 00003.jpg 51 105 968 659 145
3 00004.jpg 67 84 581 407 187
4 00005.jpg 140 151 593 339 185
... ... ... ... ... ... ...
8036 08037.jpg 49 57 1169 669 63
8037 08038.jpg 23 18 640 459 16
8038 08039.jpg 33 27 602 252 17
8039 08040.jpg 33 142 521 376 38
8040 08041.jpg 77 73 506 380 32

8041 rows × 6 columns

In [9]:
# Train images with its classification
classification=[]
category = os.listdir(train_path)
for category in category:
    for i in range(0,len(os.listdir(os.path.join(train_path,category)))):
        image = os.listdir(os.path.join(train_path,category))[i]
        classification.append([image,category])
In [10]:
#converting list to dataframe
train_classification = pd.DataFrame(classification,columns=['file','classification'])
In [11]:
train_classification
Out[11]:
file classification
0 03165.jpg Acura TL Type-S 2008
1 03243.jpg Acura TL Type-S 2008
2 07214.jpg Acura TL Type-S 2008
3 05405.jpg Acura TL Type-S 2008
4 07661.jpg Acura TL Type-S 2008
... ... ...
8054 06013.jpg Volvo XC90 SUV 2007
8055 00852.jpg Volvo XC90 SUV 2007
8056 03307.jpg Volvo XC90 SUV 2007
8057 04445.jpg Volvo XC90 SUV 2007
8058 01513.jpg Volvo XC90 SUV 2007

8059 rows × 2 columns

In [12]:
# Test images with its classification
classification2=[]
category2 = os.listdir(test_path)
for category in category2:
    for i in range(0,len(os.listdir(os.path.join(test_path,category)))):
        image = os.listdir(os.path.join(test_path,category))[i]
        classification2.append([image,category])
In [13]:
#converting list to dataframe
test_classification = pd.DataFrame(classification2,columns=['file','classification'])
In [14]:
test_classification
Out[14]:
file classification
0 00183.jpg Acura RL Sedan 2012
1 00249.jpg Acura RL Sedan 2012
2 01856.jpg Acura RL Sedan 2012
3 00347.jpg Acura RL Sedan 2012
4 00849.jpg Acura RL Sedan 2012
... ... ...
8036 07833.jpg Volvo C30 Hatchback 2012
8037 00563.jpg Volvo C30 Hatchback 2012
8038 05867.jpg Volvo C30 Hatchback 2012
8039 00338.jpg Volvo C30 Hatchback 2012
8040 02215.jpg Volvo C30 Hatchback 2012

8041 rows × 2 columns

In [15]:
# Merging the data frames
train_df_final = train_df.merge(train_classification, on="file", how = 'inner')
In [16]:
train_df_final
Out[16]:
file x_min y_min x_max y_max class classification
0 00001.jpg 39 116 569 375 14 Audi TTS Coupe 2012
1 00002.jpg 36 116 868 587 3 Acura TL Sedan 2012
2 00003.jpg 85 109 601 381 91 Dodge Dakota Club Cab 2007
3 00004.jpg 621 393 1484 1096 134 Hyundai Sonata Hybrid Sedan 2012
4 00005.jpg 14 36 133 99 106 Ford F-450 Super Duty Crew Cab 2012
... ... ... ... ... ... ... ...
8054 08140.jpg 3 44 423 336 78 Chrysler Town and Country Minivan 2012
8055 08141.jpg 138 150 706 523 196 smart fortwo Convertible 2012
8056 08142.jpg 26 246 660 449 163 Mercedes-Benz SL-Class Coupe 2009
8057 08143.jpg 78 526 1489 908 112 Ford GT Coupe 2006
8058 08144.jpg 20 240 862 677 17 Audi 100 Sedan 1994

8059 rows × 7 columns

In [17]:
#adding file path in the dataframe
train_df_final['file_path']=train_path+'/'+train_df_final['classification']+'/'+train_df_final['file']
In [18]:
train_df_final
Out[18]:
file x_min y_min x_max y_max class classification file_path
0 00001.jpg 39 116 569 375 14 Audi TTS Coupe 2012 /content/drive/MyDrive/Capstone Data/Train Ima...
1 00002.jpg 36 116 868 587 3 Acura TL Sedan 2012 /content/drive/MyDrive/Capstone Data/Train Ima...
2 00003.jpg 85 109 601 381 91 Dodge Dakota Club Cab 2007 /content/drive/MyDrive/Capstone Data/Train Ima...
3 00004.jpg 621 393 1484 1096 134 Hyundai Sonata Hybrid Sedan 2012 /content/drive/MyDrive/Capstone Data/Train Ima...
4 00005.jpg 14 36 133 99 106 Ford F-450 Super Duty Crew Cab 2012 /content/drive/MyDrive/Capstone Data/Train Ima...
... ... ... ... ... ... ... ... ...
8054 08140.jpg 3 44 423 336 78 Chrysler Town and Country Minivan 2012 /content/drive/MyDrive/Capstone Data/Train Ima...
8055 08141.jpg 138 150 706 523 196 smart fortwo Convertible 2012 /content/drive/MyDrive/Capstone Data/Train Ima...
8056 08142.jpg 26 246 660 449 163 Mercedes-Benz SL-Class Coupe 2009 /content/drive/MyDrive/Capstone Data/Train Ima...
8057 08143.jpg 78 526 1489 908 112 Ford GT Coupe 2006 /content/drive/MyDrive/Capstone Data/Train Ima...
8058 08144.jpg 20 240 862 677 17 Audi 100 Sedan 1994 /content/drive/MyDrive/Capstone Data/Train Ima...

8059 rows × 8 columns

In [19]:
test_df_final = test_df.merge(test_classification, on="file", how = 'inner')
In [20]:
test_df_final
Out[20]:
file x_min y_min x_max y_max class classification
0 00001.jpg 30 52 246 147 181 Suzuki Aerio Sedan 2007
1 00002.jpg 100 19 576 203 103 Ferrari 458 Italia Convertible 2012
2 00003.jpg 51 105 968 659 145 Jeep Patriot SUV 2012
3 00004.jpg 67 84 581 407 187 Toyota Camry Sedan 2012
4 00005.jpg 140 151 593 339 185 Tesla Model S Sedan 2012
... ... ... ... ... ... ... ...
8036 08037.jpg 49 57 1169 669 63 Chevrolet Sonic Sedan 2012
8037 08038.jpg 23 18 640 459 16 Audi V8 Sedan 1994
8038 08039.jpg 33 27 602 252 17 Audi 100 Sedan 1994
8039 08040.jpg 33 142 521 376 38 BMW Z4 Convertible 2012
8040 08041.jpg 77 73 506 380 32 BMW X5 SUV 2007

8041 rows × 7 columns

In [21]:
test_df_final['file_path']=test_path+'/'+test_df_final['classification']+'/'+test_df_final['file']
In [22]:
test_df_final
Out[22]:
file x_min y_min x_max y_max class classification file_path
0 00001.jpg 30 52 246 147 181 Suzuki Aerio Sedan 2007 /content/drive/MyDrive/Capstone Data/Test Imag...
1 00002.jpg 100 19 576 203 103 Ferrari 458 Italia Convertible 2012 /content/drive/MyDrive/Capstone Data/Test Imag...
2 00003.jpg 51 105 968 659 145 Jeep Patriot SUV 2012 /content/drive/MyDrive/Capstone Data/Test Imag...
3 00004.jpg 67 84 581 407 187 Toyota Camry Sedan 2012 /content/drive/MyDrive/Capstone Data/Test Imag...
4 00005.jpg 140 151 593 339 185 Tesla Model S Sedan 2012 /content/drive/MyDrive/Capstone Data/Test Imag...
... ... ... ... ... ... ... ... ...
8036 08037.jpg 49 57 1169 669 63 Chevrolet Sonic Sedan 2012 /content/drive/MyDrive/Capstone Data/Test Imag...
8037 08038.jpg 23 18 640 459 16 Audi V8 Sedan 1994 /content/drive/MyDrive/Capstone Data/Test Imag...
8038 08039.jpg 33 27 602 252 17 Audi 100 Sedan 1994 /content/drive/MyDrive/Capstone Data/Test Imag...
8039 08040.jpg 33 142 521 376 38 BMW Z4 Convertible 2012 /content/drive/MyDrive/Capstone Data/Test Imag...
8040 08041.jpg 77 73 506 380 32 BMW X5 SUV 2007 /content/drive/MyDrive/Capstone Data/Test Imag...

8041 rows × 8 columns

In [23]:
# Adding the height & width field with the dataframe
#train_df_final['width'] = 0
#train_df_final['height'] = 0
In [24]:
# extracting the image width & height and loading to dataframe
#for i in range(0,train_df_final['file_path'].shape[0]):
 # image = load_img(train_df_final['file_path'][i])
  #train_df_final['width'][i]=image.size[0]
  #train_df_final['height'][i]=image.size[1]
In [25]:
#train_df_final
In [26]:
#train_df_final['file_path'].shape[0]
In [27]:
#train_df_final.to_excel("train_df_final_caps.xlsx")
In [28]:
train_df_final= pd.read_excel('/content/drive/MyDrive/Capstone Data/train_df_final_caps.xlsx')
In [29]:
train_df_final
Out[29]:
file x_min y_min x_max y_max class classification file_path width height
0 00001.jpg 39 116 569 375 14 Audi TTS Coupe 2012 /content/drive/MyDrive/Capstone Data/Train Ima... 600 400
1 00002.jpg 36 116 868 587 3 Acura TL Sedan 2012 /content/drive/MyDrive/Capstone Data/Train Ima... 900 675
2 00003.jpg 85 109 601 381 91 Dodge Dakota Club Cab 2007 /content/drive/MyDrive/Capstone Data/Train Ima... 640 480
3 00004.jpg 621 393 1484 1096 134 Hyundai Sonata Hybrid Sedan 2012 /content/drive/MyDrive/Capstone Data/Train Ima... 2100 1386
4 00005.jpg 14 36 133 99 106 Ford F-450 Super Duty Crew Cab 2012 /content/drive/MyDrive/Capstone Data/Train Ima... 144 108
... ... ... ... ... ... ... ... ... ... ...
8054 08140.jpg 3 44 423 336 78 Chrysler Town and Country Minivan 2012 /content/drive/MyDrive/Capstone Data/Train Ima... 424 360
8055 08141.jpg 138 150 706 523 0 smart fortwo Convertible 2012 /content/drive/MyDrive/Capstone Data/Train Ima... 800 600
8056 08142.jpg 26 246 660 449 163 Mercedes-Benz SL-Class Coupe 2009 /content/drive/MyDrive/Capstone Data/Train Ima... 750 469
8057 08143.jpg 78 526 1489 908 112 Ford GT Coupe 2006 /content/drive/MyDrive/Capstone Data/Train Ima... 1600 1067
8058 08144.jpg 20 240 862 677 17 Audi 100 Sedan 1994 /content/drive/MyDrive/Capstone Data/Train Ima... 883 683

8059 rows × 10 columns

In [30]:
# Adding the height & width field with the dataframe
#test_df_final['width'] = 0
#test_df_final['height'] = 0
# extracting the image width & height and loading to dataframe
#for i in range(0,test_df_final['file_path'].shape[0]):
 # image = load_img(test_df_final['file_path'][i])
  #test_df_final['width'][i]=image.size[0]
  #test_df_final['height'][i]=image.size[1]
  #print(i)
In [31]:
#test_df_final
In [32]:
#test_df_final.to_excel("test_df_final_caps.xlsx")
In [33]:
test_df_final= pd.read_excel('/content/drive/MyDrive/Capstone Data/test_df_final_caps.xlsx')
In [34]:
test_df_final
Out[34]:
file x_min y_min x_max y_max class classification file_path width height
0 00001.jpg 30 52 246 147 181 Suzuki Aerio Sedan 2007 /content/drive/MyDrive/Capstone Data/Test Imag... 276 182
1 00002.jpg 100 19 576 203 103 Ferrari 458 Italia Convertible 2012 /content/drive/MyDrive/Capstone Data/Test Imag... 640 360
2 00003.jpg 51 105 968 659 145 Jeep Patriot SUV 2012 /content/drive/MyDrive/Capstone Data/Test Imag... 1024 741
3 00004.jpg 67 84 581 407 187 Toyota Camry Sedan 2012 /content/drive/MyDrive/Capstone Data/Test Imag... 640 480
4 00005.jpg 140 151 593 339 185 Tesla Model S Sedan 2012 /content/drive/MyDrive/Capstone Data/Test Imag... 600 373
... ... ... ... ... ... ... ... ... ... ...
8036 08037.jpg 49 57 1169 669 63 Chevrolet Sonic Sedan 2012 /content/drive/MyDrive/Capstone Data/Test Imag... 1200 800
8037 08038.jpg 23 18 640 459 16 Audi V8 Sedan 1994 /content/drive/MyDrive/Capstone Data/Test Imag... 640 480
8038 08039.jpg 33 27 602 252 17 Audi 100 Sedan 1994 /content/drive/MyDrive/Capstone Data/Test Imag... 635 303
8039 08040.jpg 33 142 521 376 38 BMW Z4 Convertible 2012 /content/drive/MyDrive/Capstone Data/Test Imag... 640 427
8040 08041.jpg 77 73 506 380 32 BMW X5 SUV 2007 /content/drive/MyDrive/Capstone Data/Test Imag... 640 426

8041 rows × 10 columns

In [35]:
# Display images with bounding box
img_num = np.random.randint(0,train_df_final.shape[0])
img = cv2.imread(train_df_final.loc[img_num,'file_path'])
img = cv2.resize(img,(500,500))
w = train_df_final.loc[img_num,'width']
h = train_df_final.loc[img_num,'height']
cv2.rectangle(img,
              (int(train_df_final.loc[img_num,'x_min']*500/w),int(train_df_final.loc[img_num,'y_min']*500/h)),
              (int(train_df_final.loc[img_num,'x_max']*500/w),int(train_df_final.loc[img_num,'y_max']*500/h)),(0,255,0),2)

img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
plt.suptitle(train_df_final.loc[img_num,'classification'])
plt.imshow(img)
plt.show()
In [38]:
def batch_generator(df,batch_size=32):
  while True:
    image_nums = np.random.randint(0,df.shape[0],size=batch_size)
    batch_images = np.zeros(shape=(batch_size,img_size,img_size,3))
    batch_labels = np.zeros(shape=(batch_size,196))
    batch_bboxes = np.zeros(shape=(batch_size,4))

    for i in range(batch_size):
      img=tf.keras.preprocessing.image.load_img(df.loc[image_nums[i],'file_path'],
                                                target_size=(img_size,img_size))
      
      img_array = tf.keras.preprocessing.image.img_to_array(img)
      batch_images[i] = img_array

      c1_label = df.loc[image_nums[i],'class']
      c1_label = tf.keras.utils.to_categorical(c1_label,196)
      batch_labels[i]=c1_label

      img_width = df.loc[image_nums[i],'width']
      img_height = df.loc[image_nums[i],'height']

      xmin = df.loc[image_nums[i],'x_min']*img_size/img_width
      xmax = df.loc[image_nums[i],'x_max']*img_size/img_width

      ymin = df.loc[image_nums[i],'y_min']*img_size/img_width
      ymax = df.loc[image_nums[i],'y_max']*img_size/img_width

      batch_bboxes[i]=[xmin,ymin,xmax-xmin,ymax-ymin]

      for i in range(batch_size):
        batch_images[i]=tf.keras.applications.vgg16.preprocess_input(batch_images[i])
      
      batch_bboxes=batch_bboxes/img_size

      yield batch_images,[batch_labels,batch_bboxes]
In [39]:
#building the model
img_size = 224
tf.keras.backend.clear_session()
model = tf.keras.applications.VGG16(include_top=False,
                                    input_shape=(img_size,img_size,3),weights='imagenet')
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5
58892288/58889256 [==============================] - 0s 0us/step
In [40]:
model.summary()
Model: "vgg16"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         [(None, 224, 224, 3)]     0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         
=================================================================
Total params: 14,714,688
Trainable params: 14,714,688
Non-trainable params: 0
_________________________________________________________________
In [41]:
len(model.layers)
Out[41]:
19
In [42]:
for layer in model.layers:
  layer.trainable=False
In [43]:
model.summary()
Model: "vgg16"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         [(None, 224, 224, 3)]     0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   
_________________________________________________________________
block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         
=================================================================
Total params: 14,714,688
Trainable params: 0
Non-trainable params: 14,714,688
_________________________________________________________________
In [44]:
model.output
Out[44]:
<KerasTensor: shape=(None, 7, 7, 512) dtype=float32 (created by layer 'block5_pool')>
In [45]:
x1 = model.output
x2 = tf.keras.layers.GlobalAveragePooling2D()(x1)
x3 = tf.keras.layers.Dropout(0.5)(x2)
x4 = tf.keras.layers.Dense(200,activation='relu')(x3)
x5 = tf.keras.layers.BatchNormalization()(x4)
In [46]:
label_output = tf.keras.layers.Dense(196,
                                     activation = 'softmax',
                                     name='class_op')(x5)
In [47]:
bbox_output = tf.keras.layers.Dense(4,
                                     activation = 'sigmoid',
                                     name='reg_op')(x5)
In [48]:
final_model = tf.keras.models.Model(inputs=model.input,outputs=[label_output,bbox_output])
In [49]:
def calculate_iou(y_true,y_pred):
  result = []
  for i in range(0,y_true.shape[0]):
    y_true = np.array(y_true,dtype=np.float32)
    y_pred = np.array(y_pred,dtype=np.float32)
    x_boxTrue_tleft = y_true[i,0]
    y_boxTrue_tleft = y_true[i,1]
    boxTrue_width = y_true[i,2]
    boxTrue_height = y_true[i,3]
    area_boxTrue = (boxTrue_width*boxTrue_height)

    #box_pred
    x_boxPred_tleft = y_pred[i,0]
    y_boxPred_tleft = y_pred[i,1]
    boxPred_width = y_pred[i,2]
    boxPred_height = y_pred[i,3]
    area_boxPred = (boxPred_width*boxPred_height)

    #calculating the bottom right co-ordinates
    x_boxTrue_br = x_boxTrue_tleft + boxTrue_width
    y_boxTrue_br = y_boxTrue_tleft + boxTrue_height

    x_boxPred_br = x_boxPred_tleft + boxPred_width
    y_boxPred_br = y_boxPred_tleft + boxPred_height

    #calculating the intersection box

    x_boxInt_tleft = np.max([x_boxTrue_tleft,x_boxPred_tleft])
    y_boxInt_tleft = np.max([y_boxTrue_tleft,y_boxPred_tleft])

    x_boxInt_br = np.min([x_boxTrue_br,x_boxPred_br])
    y_boxInt_br = np.min([y_boxTrue_br,y_boxPred_br])

    # calculating the area of intersection
    area_of_intersection = \
    np.max([0,(x_boxInt_br-x_boxInt_tleft)])*np.max([0,(y_boxInt_br-y_boxInt_tleft)])

    iou = area_of_intersection / ((area_boxTrue+area_boxPred)-area_of_intersection)

    iou = np.array(iou,dtype=np.float32)

    result.append(iou)

  return np.mean(result)
In [50]:
def IoU(y_true,y_pred):
  
  iou = tf.py_function(calculate_iou,[y_true,y_pred],tf.float32)
  return iou
In [51]:
final_model.compile(optimizer='adam',
                    loss = {'reg_op':'mse','class_op':'categorical_crossentropy'},
                    loss_weights={'reg_op':1,'class_op':1},
                    metrics={'reg_op':[IoU],'class_op':['accuracy']})
In [52]:
#model training
img_size = 224
batch_size = 32
train_generator=batch_generator(train_df_final,batch_size=32)
test_generator=batch_generator(test_df_final,batch_size=32)
In [53]:
def batch_generator(df,batch_size=32):
    while True:
      image_nums = np.random.randint(0,df.shape[0],size=batch_size)
      batch_images = np.zeros(shape=(batch_size,img_size,img_size,3))
      batch_labels = np.zeros(shape=(batch_size,196))
      batch_bboxes = np.zeros(shape=(batch_size,4))

      for i in range(batch_size):
        img=tf.keras.preprocessing.image.load_img(df.loc[image_nums[i],'file_path'],target_size=(img_size,img_size))
      
        img_array = tf.keras.preprocessing.image.img_to_array(img)
        batch_images[i] = img_array

        c1_label = df.loc[image_nums[i],'class']
        c1_label = tf.keras.utils.to_categorical(c1_label,196)
        batch_labels[i]=c1_label

        img_width = df.loc[image_nums[i],'width']
        img_height = df.loc[image_nums[i],'height']

        xmin = df.loc[image_nums[i],'x_min']*img_size/img_width
        xmax = df.loc[image_nums[i],'x_max']*img_size/img_width

        ymin = df.loc[image_nums[i],'y_min']*img_size/img_width
        ymax = df.loc[image_nums[i],'y_max']*img_size/img_width

        batch_bboxes[i]=[xmin,ymin,xmax-xmin,ymax-ymin]

      for i in range(batch_size):
        batch_images[i]=tf.keras.applications.vgg16.preprocess_input(batch_images[i])

      batch_bboxes=batch_bboxes/img_size

      yield batch_images,[batch_labels,batch_bboxes]
In [54]:
final_model.fit(train_generator,
                epochs=10,
                steps_per_epoch=train_df_final.shape[0]//batch_size,
                validation_data=test_generator,
                validation_steps=test_df_final.shape[0]//batch_size)
Epoch 1/10
251/251 [==============================] - 177s 555ms/step - loss: 2.2397 - class_op_loss: 2.0543 - reg_op_loss: 0.1854 - class_op_accuracy: 0.0839 - reg_op_IoU: 0.0045 - val_loss: 4.4847 - val_class_op_loss: 4.4620 - val_reg_op_loss: 0.0227 - val_class_op_accuracy: 0.0021 - val_reg_op_IoU: 0.0033
Epoch 2/10
251/251 [==============================] - 131s 525ms/step - loss: 2.4602 - class_op_loss: 2.4395 - reg_op_loss: 0.0207 - class_op_accuracy: 0.1066 - reg_op_IoU: 0.0029 - val_loss: 4.1736 - val_class_op_loss: 4.1523 - val_reg_op_loss: 0.0214 - val_class_op_accuracy: 0.0044 - val_reg_op_IoU: 0.0015
Epoch 3/10
251/251 [==============================] - 131s 523ms/step - loss: 2.4908 - class_op_loss: 2.4772 - reg_op_loss: 0.0136 - class_op_accuracy: 0.1182 - reg_op_IoU: 0.0026 - val_loss: 3.7833 - val_class_op_loss: 3.7622 - val_reg_op_loss: 0.0211 - val_class_op_accuracy: 0.0061 - val_reg_op_IoU: 7.4940e-04
Epoch 4/10
251/251 [==============================] - 130s 519ms/step - loss: 2.5808 - class_op_loss: 2.5699 - reg_op_loss: 0.0109 - class_op_accuracy: 0.1105 - reg_op_IoU: 0.0040 - val_loss: 3.9829 - val_class_op_loss: 3.8770 - val_reg_op_loss: 0.1059 - val_class_op_accuracy: 0.0019 - val_reg_op_IoU: 0.0049
Epoch 5/10
251/251 [==============================] - 127s 507ms/step - loss: 2.6098 - class_op_loss: 2.5820 - reg_op_loss: 0.0277 - class_op_accuracy: 0.0945 - reg_op_IoU: 0.0038 - val_loss: 3.6735 - val_class_op_loss: 3.6529 - val_reg_op_loss: 0.0206 - val_class_op_accuracy: 0.0072 - val_reg_op_IoU: 0.0014
Epoch 6/10
251/251 [==============================] - 127s 506ms/step - loss: 2.4430 - class_op_loss: 2.4318 - reg_op_loss: 0.0112 - class_op_accuracy: 0.0969 - reg_op_IoU: 0.0025 - val_loss: 3.1030 - val_class_op_loss: 3.0975 - val_reg_op_loss: 0.0055 - val_class_op_accuracy: 0.0128 - val_reg_op_IoU: 3.6207e-04
Epoch 7/10
251/251 [==============================] - 125s 499ms/step - loss: 2.2729 - class_op_loss: 2.2632 - reg_op_loss: 0.0097 - class_op_accuracy: 0.0938 - reg_op_IoU: 0.0024 - val_loss: 3.8202 - val_class_op_loss: 3.8138 - val_reg_op_loss: 0.0063 - val_class_op_accuracy: 0.0017 - val_reg_op_IoU: 2.1555e-05
Epoch 8/10
192/251 [=====================>........] - ETA: 14s - loss: 2.4330 - class_op_loss: 2.4238 - reg_op_loss: 0.0092 - class_op_accuracy: 0.1040 - reg_op_IoU: nan   
/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:37: RuntimeWarning: invalid value encountered in double_scalars
251/251 [==============================] - 119s 477ms/step - loss: 2.4210 - class_op_loss: 2.4113 - reg_op_loss: 0.0097 - class_op_accuracy: 0.1043 - reg_op_IoU: nan - val_loss: 3.5807 - val_class_op_loss: 3.5721 - val_reg_op_loss: 0.0086 - val_class_op_accuracy: 3.7351e-04 - val_reg_op_IoU: 4.1709e-05
Epoch 9/10
251/251 [==============================] - 119s 475ms/step - loss: 2.5540 - class_op_loss: 2.5359 - reg_op_loss: 0.0181 - class_op_accuracy: 0.1016 - reg_op_IoU: nan - val_loss: 3.5489 - val_class_op_loss: 3.5358 - val_reg_op_loss: 0.0131 - val_class_op_accuracy: 0.0055 - val_reg_op_IoU: 9.3779e-05
Epoch 10/10
251/251 [==============================] - 120s 478ms/step - loss: 2.5342 - class_op_loss: 2.5253 - reg_op_loss: 0.0089 - class_op_accuracy: 0.1023 - reg_op_IoU: nan - val_loss: 3.5933 - val_class_op_loss: 3.5120 - val_reg_op_loss: 0.0814 - val_class_op_accuracy: 0.0072 - val_reg_op_IoU: 0.0012
Out[54]:
<tensorflow.python.keras.callbacks.History at 0x7fe2943889d0>